﻿//This library allows the import and parsing of IIS log files from the filesystem to a List<IISLogEvent>.This project arose from the need to have a component that allows both the import of small log files as well as larger files(> 1Gb).
//Build with NET Standard 2.0.3, can be used with.Net Framework or .Net Core
//## Processing
//The processing engine detects the size of the file to be processed and if it is less than 50 Mb it does a single read to memory and treats the data from there, for larger files to avoid OutOfMemory, reading is done line by line. 
//## Properties
//**FilePath**[*string *] -> Path to the file
//**MissingRecords**[*bool *] -> Flag that indicates if there are any missing records.For larger files, this property can be flagged as true.  
//**CurrentFileRecord**[*int *] -> When processing large files this will store the currently file record index.For example, if the file has 1.000.000 log events and the processing is done in blocks of 250000(* MaxFileRecord2Read*), we'll have 4 cycles each on with this flag set with 250.000, 500.000, 750.000 and finally 1.000.000
//** MaxFileRecord2Read**[*int *] ->  Controls the maximum limit of items that the<IISLogEvent> List can have.If the number of events in the log file exceeds MaxFileRecord2Read the MissingRecords variable assumes the value of true and we can perform one more reading of a MaxFileRecord2Read set.
//For files less than 50Mb this value has no effect because the engine performs a single read to memory and treats the data from there.
//For example, if the file has  1.000.000 log events and this is set to 250.000 we will 4 read cycle each one extracting a  List<IISLogEvent> with a count of 250.000
//Usage : 
//           List<IISLogEvent> logs = new List<IISLogEvent>();
//            using (ParserEngine parser = new Par serEngine([filepath]))
//            {
//                while (parser.MissingRecords)
//                {
//                    logs = parser.ParseLog().ToList();
//                }
//            }
using Mrc.Entity;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;

namespace Mrc.IISLog
{
    public class ParserEngine : IDisposable
    {
        public string FilePath { get; set; }
        public bool MissingRecords { get; private set; } = true;
        public int MaxFileRecord2Read { get; set; } = 1000000;
        public int CurrentFileRecord { get; private set; }
        private readonly StreamReader _logfile;
        private string[] _headerFields;
        Hashtable dataStruct = new Hashtable();
        private readonly int _mbSize;
        private bool isGetHeader=false;
        public string time { get;set;}

        public string _fileName { get; set; }

        public Dictionary<string, string> keywordDic = new Dictionary<string, string>();
        public ParserEngine(string filePath)
        {
            
            if (File.Exists(filePath))
            {
                FilePath = filePath;

               _fileName= Path.GetFileName(filePath);
                _logfile = new StreamReader(FilePath);
                _mbSize = (int)new FileInfo(filePath).Length / 1024 / 1024;
                if (keywordDic.Count == 0)
                     keywordDic=IISManager.GetKeyWords();
            }
            else
            {
                throw new Exception($"Could not find File {filePath}");
            }
        }

        public IEnumerable<IISLogEvent> ParseLog()
        {
            if (_mbSize < 50)
            {
                return QuickProcess();
            }
            else
            {
                return LongProcess();
            }
        }

        private IEnumerable<IISLogEvent> QuickProcess()
        {
            List<IISLogEvent> events = new List<IISLogEvent>();
            var lines = File.ReadAllLines(FilePath);
            foreach (string line in lines)
            {
                ProcessLine(line, events);
            }
            MissingRecords = false;
            return events;
        }

        private IEnumerable<IISLogEvent> LongProcess()
        {
            string line;
            List<IISLogEvent> events = new List<IISLogEvent>();
            MissingRecords = false;
            while ((line = _logfile.ReadLine()) != null)
            {
                ProcessLine(line, events);
                if (events?.Count > 0 && events?.Count % MaxFileRecord2Read == 0)
                {
                    MissingRecords = true;
                    break;
                }
            }
            return events;
        }

        private void ProcessLine(string line, List<IISLogEvent> events)
        {
            if(!isGetHeader)
            {
                if (line.StartsWith("#Fields:"))
                {
                    _headerFields = line.Replace("#Fields: ", string.Empty).Split(' ');
                    isGetHeader = true;
                }
                if (line.StartsWith("#Date:"))
                {
                    time = line.Replace("#Date: ", string.Empty).Split(' ')[0];
                    
                }
            }
            if (!line.StartsWith("#") && _headerFields != null) 
            {
                string[] fieldsData = line.Split(' ');
                FillDataStruct(fieldsData, _headerFields);
                events?.Add(NewEventObj());
                CurrentFileRecord++;
            }
        }

        private IISLogEvent NewEventObj()
        {
            bool isWarning = false;
            foreach (var item in keywordDic)
            {
                if ((dataStruct["cs-uri-stem"]?.ToString().Contains(item.Value)??false) || (dataStruct["cs-uri-query"]?.ToString().Contains(item.Value)??false))
                    isWarning = true;
            }

            return new IISLogEvent
            {
                logName = _fileName,
                DateTimeEvent = GetEventDateTime(),
                sSitename = dataStruct["s-sitename"]?.ToString(),
                sComputername = dataStruct["s-computername"]?.ToString(),
                sIp = dataStruct["s-ip"]?.ToString(),
                csMethod = dataStruct["cs-method"]?.ToString(),
                csUriStem = dataStruct["cs-uri-stem"]?.ToString(),
                csUriQuery = dataStruct["cs-uri-query"]?.ToString(),
                sPort = dataStruct["s-port"] != null ? int.Parse(dataStruct["s-port"]?.ToString()) : (int?)null,
                csUsername = dataStruct["cs-username"]?.ToString(),
                cIp = dataStruct["c-ip"]?.ToString(),
                csVersion = dataStruct["cs-version"]?.ToString(),
                csUserAgent = dataStruct["cs(User-Agent)"]?.ToString(),
                csCookie = dataStruct["cs(Cookie)"]?.ToString(),
                csReferer = dataStruct["cs(Referer)"]?.ToString(),
                csHost = dataStruct["cs-host"]?.ToString(),
                scStatus = dataStruct["sc-status"] != null ? int.Parse(dataStruct["sc-status"]?.ToString()) : (int?)null,
                scSubstatus = dataStruct["sc-substatus"] != null ? int.Parse(dataStruct["sc-substatus"]?.ToString()) : (int?)null,
                scWin32Status = dataStruct["sc-win32-status"] != null ? long.Parse(dataStruct["sc-win32-status"]?.ToString()) : (long?)null,
                scBytes = dataStruct["sc-bytes"] != null ? int.Parse(dataStruct["sc-bytes"]?.ToString()) : (int?)null,
                csBytes = dataStruct["cs-bytes"] != null ? int.Parse(dataStruct["cs-bytes"]?.ToString()) : (int?)null,
                timeTaken = dataStruct["time-taken"] != null ? int.Parse(dataStruct["time-taken"]?.ToString()) : (int?)null,
                isWarning=isWarning
            };
        }

        private DateTime GetEventDateTime()
        {
            DateTime finalDate = DateTime.Parse($"{dataStruct["date"]} {dataStruct["time"]}").AddHours(8);//北京时间
            return finalDate;
        }
        private void FillDataStruct(string[] fieldsData, string[] header)
        {
            dataStruct.Clear();
            for (int i = 0; i < header.Length; i++)
            {
                dataStruct.Add(header[i], fieldsData[i] == "-" ? null : fieldsData[i]);
            }
        }
        public void Dispose()
        {
            _logfile?.Close();
        }
    }

}
